
import { ProxyModel } from "../model"
import superAgent from 'superagent'
import { MongoDBClient } from "../db"
import geoip from 'geoip-lite'
import { getCancelTokenSource } from "../utils/proxy-util"
import axios, { AxiosResponse } from "axios"
import { HttpsProxyAgent } from "https-proxy-agent"
// import { ExtractProxy, LoadProxy, TransformProxy } from "./interface-proxy"
import { ExtractClientInterface, ProxyInterface } from "../etl"
import { getLogger } from "../utils/log4j"
const logger = getLogger(`github proxies`)

export class GithubProxy implements ExtractClientInterface {

	constructor(
		protected proxyResourceUrls: string[],
	) { }
	async * extract(): AsyncGenerator<ProxyInterface, void, any> {

		for (const url of this.proxyResourceUrls) {
			let res: AxiosResponse<never>
			const source = getCancelTokenSource()
			setTimeout(() => {
				source.cancel(`请求超时`)
			}, 10000);
			try {
				logger.info(`${url}`)
				res = await axios.get(url, { cancelToken: source.token })
			} catch (error) {
				logger.error(`request ${url} fail`)
				continue
			}
			const proxies = (res.data as string).split('\n')
			for (const item of proxies) {
				const proxySpilt = item.split(':')
				const ip = proxySpilt[0]
				const port = proxySpilt[1]
				if (!ip || !port) {
					logger.error(`proxy wrong format or empty value`)
					continue
				}
				const proxy: ProxyInterface = {
					ip: ip,
					port: Number(port)
				}
				yield proxy
			}
		}
	}
}
